{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ROC曲线\n",
    "> ROC:Receiver Operation Characteristic Curve 描述TPR和FPR之间的关系"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TPR即召回率"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![TPR](images/TPR.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FPR\n",
    "![FPR](images/FPR.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9755555555555555"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "digits = datasets.load_digits() # 加载手写数字识别数据集\n",
    "X = digits.data\n",
    "y = digits.target.copy()\n",
    "# 多分类问题转换为二分类问题，即等于9和不等于9,数据比例大约是1:9,也就是说我们只要全认为是非9，按照传统计算正确率的方法我们也有90%的正确率\n",
    "y[digits.target==9] = 1\n",
    "y[digits.target!=9] = 0\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666)\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "log_reg = LogisticRegression()\n",
    "log_reg.fit(X_train, y_train)\n",
    "log_reg.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "decision_scores = log_reg.decision_function(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 所有的边界阈值\n",
    "thresholds = np.arange(decision_scores.min(), decision_scores.max())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 计算TPR和FPR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from playML.metrics import FPR, TPR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "fprs = []\n",
    "tprs = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 循环所有的边界阈值\n",
    "for threshold in thresholds:\n",
    "    y_predict = np.array(decision_scores >= threshold, dtype='int')\n",
    "    fprs.append(FPR(y_test, y_predict))\n",
    "    tprs.append(TPR(y_test, y_predict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAASXUlEQVR4nO3de5CddX3H8feXXCGEBEi4mAsLNighasUVsNqChWpIO6S2ShNLvUUZbLHT0dqh1aEOOp2pTqVlhqqZlnqhiqgzGm0UFaEgEswiF0koZA1IVpBsIGAC5rLk2z/OwVk2JzlPyNk9e37n/ZrZmefy2+f5/nLOfvjxO8/znMhMJEmd75B2FyBJag0DXZIKYaBLUiEMdEkqhIEuSYWY2K4Tz5o1K3t6etp1eknqSHfccceWzJzdaF/bAr2np4e+vr52nV6SOlJE/Hxf+5xykaRCGOiSVAgDXZIKYaBLUiEMdEkqRNNAj4irI2JzRNy7j/0REVdGRH9E3BMRp7W+TElSM1VG6J8FFu9n/3nAgvrPRcCnDr4sSdKBanodembeHBE9+2myFPh81p7DuyYiZkbE8Zn5aItqVItteGwb37n3l+x+dk+7S5G60jmnHMsr5s1s+XFbcWPRHGDTsPWB+ra9Aj0iLqI2imf+/PktOLWq2rMn+d8HBrn61ge5ZcMWACLaXJTUpY45Yuq4DfRGsdDwWzMycyWwEqC3t9dv1hgDz+wa4mt3DPBftz7Exi1Pc+wRU/jgG1/C8tPnc9S0ye0uT1ILtSLQB4B5w9bnAo+04Lg6CL948td8/kcP8aUfP8yvdgzx8rkz+Ldlv815i45n8kQvbpJK1IpAXwVcEhHXAmcATzl/3h6ZyU8e3srVP3yI76z7JQCLTz2Od72uh9PmH0k4xyIVrWmgR8SXgLOBWRExAPwjMAkgMz8NrAaWAP3AM8A7R6tYNZaZ3LJhC1d8/wHufPhJjpg6kXf/7om87TU9zJl5aLvLkzRGqlzlsrzJ/gT+qmUV6YD86GdbuOJ7D7D2oa28aMZULl96Km9+1VwOm9y2B2lKahP/6jvU2oee4JPffYDbNj7OsUdM4aNLT+WCV89jysQJ7S5NUpsY6B3mzoe38snvPcAtG7Yw6/ApXPZHC3nrGfOZOskgl7qdgb4P/Zu38bPBp9tdxm8MPZt87ScD/OD/NnPUtMn8w5KX8hdn9nDoZINcUo2B3sAN9z3Gxdfcwe5nx9el8jMPm8TfLX4Jb39ND9Om+NJJej5TYYSb7t/Me6/5CaccfwQf++NFTDhk/Fzq13P0NINc0j6ZDsP8cMMWLvrCHSw49nC+8K4zmHHYpHaXJEmVectg3Y9+toUVn1vLSbOmcc0Kw1xS5+nqEfqqux/ha3cMAPDjB5/ghKMP47/ffQZH+owTSR2oawN997N7uPyb64mAF808lNctmMU/vellHH34lHaXJkkvSNcG+nfXPcaW7Tu5+h29/P5Lj213OZJ00Lp2Dv2aNT9nzsxDOevkY9pdiiS1RFcGev/m7dy28XHeesb8cXVZoiQdjK4M9C/e/jATDwku6J3XvLEkdYiuC/Qdu5/lq3ds4o2LjmP2dD8AlVSOrgv0b979CL/aMcSFZ5zQ7lIkqaW6LtCvuf1hXjx7GmeedFS7S5GkluqqQP/pwFPcvelJ/vyME/w6NknF6ZpA375ziL/9yt3MOHQSf3ra3HaXI0kt1xU3Fu3Zk7z/y3fRP7idz73zdJ/TIqlIXTFC/9cbNvDd9Y/xoSWn8LoFs9pdjiSNiuID/X/ueZQrb9jAW141l3e+tqfd5UjSqCliymVw207+5FO3sm3H0F77tu0Y4rT5M/nYmxb5QaikohUR6Bse28amJ37NeYuO45gRNwsdOnkiK153IlMm+t2bkspWRKA//vQuAP7m3JN5yXHT21yNJLVHEXPoT9QD/Si/mEJSFysi0J8boR/p5YiSulgRgf7E0zuZedgkJk4oojuS9IIUkYBPPL3L6RZJXa+IQH98+y6ONtAldbkiAn3rM47QJanjAz0z2bJ9F0dN88sqJHW3jg/0WzZs4Ymnd/GKuTPaXYoktVWlQI+IxRFxf0T0R8SlDfbPj4gbI+LOiLgnIpa0vtS9ZSZX3rCB42dM5U2nzRmLU0rSuNU00CNiAnAVcB6wEFgeEQtHNPswcF1mvhJYBvx7qwtt5LaNj9P3861cfNaLvbVfUterMkI/HejPzI2ZuQu4Flg6ok0CR9SXZwCPtK7ExrbvHOKfVt/HMdOn8Gevnjfap5Okca/Ks1zmAJuGrQ8AZ4xo8xHguxHxPmAacG5LqtuPN15xM7948td8+A9PYeokR+eSVGWE3uiZszlifTnw2cycCywBvhARex07Ii6KiL6I6BscHDzwaofZvG0Hc488lLf/Ts9BHUeSSlEl0AeA4XMac9l7SmUFcB1AZt4GTAX2+mqgzFyZmb2Z2Tt79uwXVnHdpAmHcN6i45jk7f6SBFQL9LXAgog4MSImU/vQc9WINg8D5wBExCnUAv3ghuCSpAPSNNAzcwi4BLgeuI/a1SzrIuLyiDi/3uwDwHsi4m7gS8A7MnPktIwkaRRV+oKLzFwNrB6x7bJhy+uB17a2NEnSgXACWpIKYaBLUiEMdEkqhIEuSYUw0CWpEAa6JBXCQJekQhjoklQIA12SCmGgS1IhDHRJKoSBLkmFMNAlqRAGuiQVwkCXpEIY6JJUCANdkgrRkYGemQztSQ6JaHcpkjRudGSgP/rUDnYN7WHeUYe1uxRJGjc6MtA3bN4OwIJjDm9zJZI0fnRkoPc/F+jHTm9zJZI0fnRooG/j6GmTOWra5HaXIknjRkcG+i+f2sHxM6e2uwxJGlc6MtABr3CRpBE6NtAlSc9noEtSIQx0SSqEgS5JhTDQJakQBrokFcJAl6RCGOiSVAgDXZIKUSnQI2JxRNwfEf0Rcek+2lwQEesjYl1EfLG1ZUqSmpnYrEFETACuAv4AGADWRsSqzFw/rM0C4O+B12bm1og4ZrQKliQ1VmWEfjrQn5kbM3MXcC2wdESb9wBXZeZWgMzc3NoyJUnNVAn0OcCmYesD9W3DnQycHBG3RsSaiFjc6EARcVFE9EVE3+Dg4AurWJLUUJVAb/RYwxyxPhFYAJwNLAf+IyJm7vVLmSszszcze2fPnn2gtUqS9qNKoA8A84atzwUeadDmG5m5OzMfBO6nFvCSpDFSJdDXAgsi4sSImAwsA1aNaPN14PUAETGL2hTMxlYWKknav6aBnplDwCXA9cB9wHWZuS4iLo+I8+vNrgcej4j1wI3ABzPz8dEqWpK0t6aXLQJk5mpg9Yhtlw1bTuD99R9JUht4p6gkFcJAl6RCGOiSVAgDXZIKYaBLUiEMdEkqhIEuSYUw0CWpEAa6JBXCQJekQhjoklQIA12SCtGRgf5sQkSj792QpO7VkYH+9M4hpk+p9KBISeoaHRno23cMcbiBLknP05mBvnOIw6ca6JI0XEcG+rYdux2hS9IIHRfomVkboRvokvQ8HRfov979LHsSp1wkaYSOC/TtO4YAHKFL0ggdF+jbdtYCfbojdEl6no4LdEfoktRY5wX6TgNdkhrpuEDf9twI3SkXSXqejgv050bo06dManMlkjS+dF6g79gNOEKXpJE6LtB3Du0BYMrEjitdkkZVx6aiT8+VpOfr2ECXJD2fgS5JhTDQJakQBrokFaJSoEfE4oi4PyL6I+LS/bR7c0RkRPS2rkRJUhVNAz0iJgBXAecBC4HlEbGwQbvpwF8Dt7e6SElSc1VG6KcD/Zm5MTN3AdcCSxu0+yjwcWBHC+uTJFVUJdDnAJuGrQ/Ut/1GRLwSmJeZ39rfgSLioojoi4i+wcHBAy5WkrRvVQK90S08+ZudEYcAVwAfaHagzFyZmb2Z2Tt79uzqVUqSmqoS6APAvGHrc4FHhq1PBxYBN0XEQ8CZwCo/GJWksVUl0NcCCyLixIiYDCwDVj23MzOfysxZmdmTmT3AGuD8zOwblYolSQ01DfTMHAIuAa4H7gOuy8x1EXF5RJw/2gVKkqqp9AzazFwNrB6x7bJ9tD374MuSJB0o7xSVpEIY6JJUCANdkgphoEtSIQx0SSqEgS5JhTDQJakQBrokFcJAl6RCGOiSVAgDXZIKYaBLUiEMdEkqhIEuSYUw0CWpEAa6JBXCQJekQhjoklQIA12SCmGgS1IhDHRJKoSBLkmFMNAlqRAGuiQVwkCXpEIY6JJUCANdkgphoEtSIQx0SSqEgS5JhTDQJakQBrokFcJAl6RCVAr0iFgcEfdHRH9EXNpg//sjYn1E3BMRN0TECa0vVZK0P00DPSImAFcB5wELgeURsXBEszuB3sx8OfBV4OOtLlSStH9VRuinA/2ZuTEzdwHXAkuHN8jMGzPzmfrqGmBua8uUJDVTJdDnAJuGrQ/Ut+3LCuDbjXZExEUR0RcRfYODg9WrlCQ1VSXQo8G2bNgw4kKgF/hEo/2ZuTIzezOzd/bs2dWrlCQ1NbFCmwFg3rD1ucAjIxtFxLnAh4CzMnNna8qTJFVVZYS+FlgQESdGxGRgGbBqeIOIeCXwGeD8zNzc+jIlSc00DfTMHAIuAa4H7gOuy8x1EXF5RJxfb/YJ4HDgKxFxV0Ss2sfhJEmjpMqUC5m5Glg9Yttlw5bPbXFdkqQD5J2iklQIA12SCmGgS1IhDHRJKoSBLkmFMNAlqRAGuiQVwkCXpEIY6JJUCANdkgphoEtSIQx0SSqEgS5JhTDQJakQBrokFcJAl6RCGOiSVAgDXZIKYaBLUiEMdEkqhIEuSYUw0CWpEAa6JBXCQJekQhjoklQIA12SCtFxgX7irGksedlxHBLR7lIkaVyZ2O4CDtQbTj2ON5x6XLvLkKRxp+NG6JKkxgx0SSqEgS5JhTDQJakQlQI9IhZHxP0R0R8RlzbYPyUivlzff3tE9LS6UEnS/jUN9IiYAFwFnAcsBJZHxMIRzVYAWzPzt4ArgH9udaGSpP2rMkI/HejPzI2ZuQu4Flg6os1S4HP15a8C50R4obgkjaUqgT4H2DRsfaC+rWGbzBwCngKOHnmgiLgoIvoiom9wcPCFVSxJaqjKjUWNRtr5AtqQmSuBlQARMRgRP69w/kZmAVte4O92KvvcHexzdziYPp+wrx1VAn0AmDdsfS7wyD7aDETERGAG8MT+DpqZsyucu6GI6MvM3hf6+53IPncH+9wdRqvPVaZc1gILIuLEiJgMLANWjWizCnh7ffnNwA8yc68RuiRp9DQdoWfmUERcAlwPTACuzsx1EXE50JeZq4D/BL4QEf3URubLRrNoSdLeKj2cKzNXA6tHbLts2PIO4C2tLW2/Vo7hucYL+9wd7HN3GJU+hzMjklQGb/2XpEIY6JJUiHEd6N34DJkKfX5/RKyPiHsi4oaI2Oc1qZ2iWZ+HtXtzRGREdPwlblX6HBEX1F/rdRHxxbGusdUqvLfnR8SNEXFn/f29pB11tkpEXB0RmyPi3n3sj4i4sv7vcU9EnHbQJ83McflD7YqanwEnAZOBu4GFI9r8JfDp+vIy4MvtrnsM+vx64LD68nu7oc/1dtOBm4E1QG+76x6D13kBcCdwZH39mHbXPQZ9Xgm8t768EHio3XUfZJ9/DzgNuHcf+5cA36Z2Y+aZwO0He87xPELvxmfINO1zZt6Ymc/UV9dQu9Grk1V5nQE+Cnwc2DGWxY2SKn1+D3BVZm4FyMzNY1xjq1XpcwJH1JdnsPcNjB0lM29m/zdYLgU+nzVrgJkRcfzBnHM8B3rLniHTQar0ebgV1P4L38ma9jkiXgnMy8xvjWVho6jK63wycHJE3BoRayJi8ZhVNzqq9PkjwIURMUDtMun3jU1pbXOgf+9NjecviW7ZM2Q6SOX+RMSFQC9w1qhWNPr22+eIOITaI5nfMVYFjYEqr/NEatMuZ1P7v7BbImJRZj45yrWNlip9Xg58NjP/JSJeQ+1mxUWZuWf0y2uLlufXeB6hH8gzZKj6DJlxrkqfiYhzgQ8B52fmzjGqbbQ06/N0YBFwU0Q8RG2ucVWHfzBa9b39jczcnZkPAvdTC/hOVaXPK4DrADLzNmAqtYdYlarS3/uBGM+B3o3PkGna5/r0w2eohXmnz6tCkz5n5lOZOSszezKzh9rnBudnZl97ym2JKu/tr1P7AJyImEVtCmbjmFbZWlX6/DBwDkBEnEIt0Et+zvYq4G31q13OBJ7KzEcP6ojt/iS4yafES4AHqH06/qH6tsup/UFD7QX/CtAP/Bg4qd01j0Gfvw88BtxV/1nV7ppHu88j2t5Eh1/lUvF1DuCTwHrgp8Cydtc8Bn1eCNxK7QqYu4A3tLvmg+zvl4BHgd3URuMrgIuBi4e9xlfV/z1+2or3tbf+S1IhxvOUiyTpABjoklQIA12SCmGgS1IhDHRJKoSBLkmFMNAlqRD/DzqvOi+B5/I7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fprs, tprs)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scikit-learn中的ROC\n",
    "+ ROC最主要用于比较两个模型孰优孰劣\n",
    "+ ROC曲线下的面积越大代表模型越好\n",
    "+ ROC曲线对极度偏斜的数据并不敏感，要比精准率-召回率曲线好很多"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "fprs, tprs, thresholds = roc_curve(y_test, decision_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAO80lEQVR4nO3db4xcV33G8e9D3BRVTaDURgr+g43qSCxRRdAqDkIqQdDKiYT9hiIbRZQqwoU29AWoUiqqgMKrglokJLdgtYiCFELgBbGQUaTSRFQIu1kUE7AjV1sHkk2iZiFpeIEgRP31xYzJdD27c9ee3dk5+/1IK8299+zM73hmn5ycc+/cVBWSpOn3skkXIEkaDwNdkhphoEtSIwx0SWqEgS5JjdgyqRfeunVr7d69e1IvL0lT6Xvf+95PqmrbsGMTC/Tdu3czNzc3qZeXpKmU5MfLHXPKRZIaYaBLUiMMdElqhIEuSY0w0CWpESMDPcnnkzyT5IfLHE+SzySZT/JIkjeNv0xJ0ihdRuhfAPavcPxmYG//5wjwj5dfliRptUaeh15V306ye4UmB4EvVu97eE8meWWSa6rq6THVqEt096nHue/0k5MuQ9ISM6+5mo+98w1jf95xzKFvB54Y2F7o77tIkiNJ5pLMLS4ujuGltZL7Tj/J2ad/NukyJK2TcVwpmiH7ht41o6qOAccAZmdnvbPGOpi55mq+8mdvnnQZktbBOEboC8DOge0dwFNjeF5J0iqMI9CPA+/tn+1yI/C88+eStP5GTrkk+TJwE7A1yQLwMeA3AKrqs8AJ4BZgHvg58KdrVaxWt9B59umfMXPN1WtckaSNostZLodHHC/gL8ZWkVZ0YaGzS1DPXHM1B984dH1aUoMm9vW5unQudEoaxkv/JakRjtAHTMOFOM6LS1qOI/QB03AhjvPikpbjCH0J56clTStH6JLUiE07Qh82X+78tKRptmlH6MPmy52fljTNNu0IHZwvl9SWTTtCl6TWGOiS1IhNNeUyuBDqAqik1myqEfrgQqgLoJJas6lG6OBCqKR2baoRuiS1rPkRuvPmkjaL5kfozptL2iyaH6GD8+aSNofmR+iStFkY6JLUiGamXJa725ALoZI2i2ZG6MvdbciFUEmbRTMjdHDxU9Lm1swIXZI2OwNdkhphoEtSIwx0SWqEgS5JjZjqs1z84i1JeslUj9D94i1JeslUj9DBc88l6YJOI/Qk+5OcSzKf5I4hx3cleSDJw0keSXLL+EuVJK1kZKAnuQI4CtwMzACHk8wsafY3wL1VdT1wCPiHcRcqSVpZlxH6DcB8VZ2vqheAe4CDS9oUcGFF8hXAU+Mrcbi7Tz3OqceeXeuXkaSp0SXQtwNPDGwv9PcN+jhwa5IF4ATwoWFPlORIkrkkc4uLi5dQ7ksunN3iQqgk9XQJ9AzZV0u2DwNfqKodwC3Al5Jc9NxVdayqZqtqdtu2bauvdol9e17Fe/btuuznkaQWdAn0BWDnwPYOLp5SuQ24F6Cqvgu8HNg6jgIlSd10CfSHgL1J9iS5kt6i5/ElbR4H3g6Q5PX0Av3y5lQkSasyMtCr6kXgduB+4FF6Z7OcSXJXkgP9Zh8B3p/k+8CXgfdV1dJpGUnSGup0YVFVnaC32Dm4786Bx2eBt4y3NEnSakz1pf+SpJcY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEVMZ6HefepxTjz076TIkaUOZykC/7/STABx84/YJVyJJG8dUBjrAvj2v4j37dk26DEnaMKY20CVJ/5+BLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNaJToCfZn+RckvkkdyzT5t1JziY5k+Tu8ZYpSRply6gGSa4AjgJ/CCwADyU5XlVnB9rsBf4aeEtVPZfk1WtVsCRpuC4j9BuA+ao6X1UvAPcAB5e0eT9wtKqeA6iqZ8ZbpiRplC6Bvh14YmB7ob9v0LXAtUm+k+Rkkv3DnijJkSRzSeYWFxcvrWJJ0lBdAj1D9tWS7S3AXuAm4DDwT0leedEvVR2rqtmqmt22bdtqa5UkraBLoC8AOwe2dwBPDWlzX1X9qqoeA87RC3hJ0jrpEugPAXuT7ElyJXAIOL6kzdeBtwEk2UpvCub8OAuVJK1sZKBX1YvA7cD9wKPAvVV1JsldSQ70m90P/DTJWeAB4K+q6qdrVbQk6WIjT1sEqKoTwIkl++4ceFzAh/s/kqQJ8EpRSWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1YuoC/e5Tj3PqsWcnXYYkbThTF+j3nX4SgINv3D7hSiRpY5m6QAfYt+dVvGffrkmXIUkbylQGuiTpYga6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmN6BToSfYnOZdkPskdK7R7V5JKMju+EiVJXYwM9CRXAEeBm4EZ4HCSmSHtrgL+Ejg17iIlSaN1GaHfAMxX1fmqegG4Bzg4pN0ngE8CvxhjfZKkjroE+nbgiYHthf6+X0tyPbCzqr6x0hMlOZJkLsnc4uLiqouVJC2vS6BnyL769cHkZcCngY+MeqKqOlZVs1U1u23btu5VSpJG6hLoC8DOge0dwFMD21cB1wEPJvkRcCNw3IVRSVpfXQL9IWBvkj1JrgQOAccvHKyq56tqa1XtrqrdwEngQFXNrUnFkqShRgZ6Vb0I3A7cDzwK3FtVZ5LcleTAWhcoSepmS5dGVXUCOLFk353LtL3p8suSJK2WV4pKUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRnQK9CT7k5xLMp/kjiHHP5zkbJJHknwryWvHX6okaSUjAz3JFcBR4GZgBjicZGZJs4eB2ar6feBrwCfHXagkaWVdRug3APNVdb6qXgDuAQ4ONqiqB6rq5/3Nk8CO8ZYpSRqlS6BvB54Y2F7o71vObcA3hx1IciTJXJK5xcXF7lVKkkbqEugZsq+GNkxuBWaBTw07XlXHqmq2qma3bdvWvUpJ0khbOrRZAHYObO8AnlraKMk7gI8Cb62qX46nPElSV11G6A8Be5PsSXIlcAg4PtggyfXA54ADVfXM+MuUJI0yMtCr6kXgduB+4FHg3qo6k+SuJAf6zT4F/Dbw1SSnkxxf5ukkSWuky5QLVXUCOLFk350Dj98x5rokSavklaKS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDWiU6An2Z/kXJL5JHcMOf6bSb7SP34qye5xFypJWtnIQE9yBXAUuBmYAQ4nmVnS7Dbguar6PeDTwN+Ou1BJ0sq6jNBvAOar6nxVvQDcAxxc0uYg8C/9x18D3p4k4ytTkjTKlg5ttgNPDGwvAPuWa1NVLyZ5Hvhd4CeDjZIcAY4A7Nq165IKnnnN1Zf0e5LUui6BPmykXZfQhqo6BhwDmJ2dveh4Fx975xsu5dckqXldplwWgJ0D2zuAp5Zrk2QL8Arg2XEUKEnqpkugPwTsTbInyZXAIeD4kjbHgT/pP34X8G9VdUkjcEnSpRk55dKfE78duB+4Avh8VZ1JchcwV1XHgX8GvpRknt7I/NBaFi1JuliXOXSq6gRwYsm+Owce/wL44/GWJklaDa8UlaRGGOiS1AgDXZIaYaBLUiMyqbMLkywCP77EX9/KkqtQNwH7vDnY583hcvr82qraNuzAxAL9ciSZq6rZSdexnuzz5mCfN4e16rNTLpLUCANdkhoxrYF+bNIFTIB93hzs8+awJn2eyjl0SdLFpnWELklawkCXpEZs6EDfjDen7tDnDyc5m+SRJN9K8tpJ1DlOo/o80O5dSSrJ1J/i1qXPSd7df6/PJLl7vWsctw6f7V1JHkjycP/zfcsk6hyXJJ9P8kySHy5zPEk+0//3eCTJmy77RatqQ/7Q+6re/wJeB1wJfB+YWdLmz4HP9h8fAr4y6brXoc9vA36r//iDm6HP/XZXAd8GTgKzk657Hd7nvcDDwO/0t1896brXoc/HgA/2H88AP5p03ZfZ5z8A3gT8cJnjtwDfpHfHtxuBU5f7mht5hL4Zb049ss9V9UBV/by/eZLeHaSmWZf3GeATwCeBX6xncWukS5/fDxytqucAquqZda5x3Lr0uYALNw1+BRffGW2qVNW3WfnObQeBL1bPSeCVSa65nNfcyIE+7ObU25drU1UvAhduTj2tuvR50G30/gs/zUb2Ocn1wM6q+sZ6FraGurzP1wLXJvlOkpNJ9q9bdWujS58/DtyaZIHe/Rc+tD6lTcxq/95H6nSDiwkZ282pp0jn/iS5FZgF3rqmFa29Ffuc5GXAp4H3rVdB66DL+7yF3rTLTfT+L+zfk1xXVf+zxrWtlS59Pgx8oar+Lsmb6d0F7bqq+t+1L28ixp5fG3mEvhlvTt2lzyR5B/BR4EBV/XKdalsro/p8FXAd8GCSH9Gbazw+5QujXT/b91XVr6rqMeAcvYCfVl36fBtwL0BVfRd4Ob0vsWpVp7/31djIgb4Zb049ss/96YfP0QvzaZ9XhRF9rqrnq2prVe2uqt301g0OVNXcZModiy6f7a/TWwAnyVZ6UzDn17XK8erS58eBtwMkeT29QF9c1yrX13Hgvf2zXW4Enq+qpy/rGSe9EjxilfgW4D/prY5/tL/vLnp/0NB7w78KzAP/Abxu0jWvQ5//Ffhv4HT/5/ika17rPi9p+yBTfpZLx/c5wN8DZ4EfAIcmXfM69HkG+A69M2BOA3806Zovs79fBp4GfkVvNH4b8AHgAwPv8dH+v8cPxvG59tJ/SWrERp5ykSStgoEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGvF/IC5d6mFx4nMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fprs, tprs)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ROC曲线下面的面积计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9830452674897119"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_test, decision_scores) # ROC曲线下的面积，ROC"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
